require 'rubygems'
require 'hpricot'
require 'open-uri'

class Util
	# patch hpricot magic 16384 bug
	def self.hpricot(uri)
		f = open(uri)
		return Hpricot(f.readlines.join)
	end
	
	def self.find_key(map, origin)
		cur = nil
		max = 0
		map.each do |key,value|
			return key if key == origin
			
			exp = Regexp.compile key
			if exp.match origin
				if max < key.size
					max = key.size
					cur = key
				end
			end 
		
		end
		return cur;
	end
	
	def self.find_data_by_from_sina(url, table_name, map)
    doc = Util.hpricot(url)
    index = {}
    result = {}

    100000.times do |i|
      table = Table.new(:doc => doc, :path=>"##{table_name}#{i} tbody tr")
      break if table.row_size < 1 || table.column_size < 1

      if index.empty?
        table.row_size.times do |r|
          next if r == 0
          origin = Iconv.conv("UTF-8", "GB2312", table[r][0])
          
          key = map[Util.find_key(map, Iconv.conv("UTF-8", "GB2312", table[r][0]))]
          index[r] = key unless key.nil?
        end
      end

      table.column_size.times do |c|
        next if c == 0
        key = table[0][c]
        value = {}
        table.row_size.times do |r|
          next unless index[r]
          value[index[r]] = table[r][c].andand.gsub(/,/, '').to_i unless table[r][c] == '--'
        end
        result[key] = value
      end

    end

    return result
  end
end
